package com.dan.util;

import java.io.*;
import java.net.*;
import java.sql.*;

public class Download {
	public static String downloadWebsite = "http://tieba.baidu.com/f?ie=utf-8&kw=%E5%A7%90%E8%84%B1";
	private static int PageNumber = 1;

	/**
	 * @param args
	 * @throws SQLException
	 * @throws IOException
	 * @throws Exception
	 */
	public static void main(String[] args) throws SQLException, IOException {
		try {
			Class.forName("com.mysql.jdbc.Driver");
		} catch (ClassNotFoundException e) {
			System.out.println("找不到MySql驱动程序包");
		}
		String connUrl = "jdbc:mysql://localhost:3306/tieba?user=root&password=abc";
		Connection con = DriverManager.getConnection(connUrl);
		Statement stmt = con.createStatement();
		ResultSet rs = null;
		ResultSet rsweb = null;
		
		// 下载主界面
		System.out.println("--------------------开始下载文件------------------------------");
		httpDownload(downloadWebsite, "jt.html");
		System.out.println("--------------------结束下载文件------------------------------");
		
		
		System.out.println("--------------------开始解析文件------------------------------");
		String[] s = readFileLine("jt.html");
		System.out.println("--------------------结束解析文件------------------------------");
		
		
		int i = 0;
		// 处理贴吧每一个贴子
		for (String page : s) {
			if (page != null) {
				i++;
				// 处理每一个贴子的第几页
				for (int j = 1; j <= PageNumber; j++) {
					boolean hasExists = false;
					String PageAddress = page + "?see_lz=1&pn=" + j;
					rsweb = stmt
							.executeQuery("select url from download where url='"
									+ PageAddress + "'");
					while (rsweb.next()) {
						hasExists = true;
					}
					// 如果网页不存在或者是第一页，或者是最后一页时，都进行下载
					if (hasExists == false || j == 1 || j == PageNumber) {
						System.out.printf("hasExists:%s,j:%s,PageNumber:%s,\n",
								hasExists, j, PageNumber);
						System.out.print("准备下载第" + i + "个帖子的第" + (j) + "页:");
						System.out.println(PageAddress);

						httpDownload(PageAddress, getLastStringFromURL(page));
						stmt.execute("insert into download (url) values ('"
								+ PageAddress + "')");

						String[] s3 = readDetailsFileLine(getLastStringFromURL(page));
						// 如果现在是第一页但不是最后一页，则进行下载
						if ((PageNumber > 1 && j == 1)) {
							continue;
						}
						for (String imglink : s3) {
							if (imglink != null && (!imglink.equals(""))) {
								boolean exist = false;
								String query = "select url from download where url='"
										+ imglink + "'";
								rs = stmt.executeQuery(query);
								while (rs.next()) {
									exist = true;
									System.out.println("图片已经存在！不需要下载");
								}

								if (exist == false) {
									httpDownload(imglink,
											getLastStringFromURL(imglink));
									stmt.execute("insert into download (url) values ('"
											+ imglink + "')");
								}
							}
						}

						File file = new File(getLastStringFromURL(page));
						if (file.exists()) {
							file.delete();
						}
						if (rsweb != null)
							rsweb.close();
					}
				}
			}
		}
		rs.close();
		stmt.close();
		con.close();
	}

	/**
	 * 使用http下载文件
	 * 
	 * @param httpUrl
	 * @param saveFile
	 * @return
	 */
	public static boolean httpDownload(String httpUrl, String saveFile) {
		// 下载网络文件
		int byteread = 0;

		URL url = null;
		try {
			url = new URL(httpUrl);
		} catch (MalformedURLException e1) {
			e1.printStackTrace();
			return false;
		}

		try {
			URLConnection conn = url.openConnection();
			InputStream inStream = conn.getInputStream();
			FileOutputStream fs = new FileOutputStream(saveFile);

			byte[] buffer = new byte[1024];
			while ((byteread = inStream.read(buffer)) != -1) {
				fs.write(buffer, 0, byteread);
			}
			fs.close();
			return true;
		} catch (FileNotFoundException e) {
			e.printStackTrace();
			return false;
		} catch (IOException e) {
			e.printStackTrace();
			return false;
		}
	}

	/**
	 * 使用BufferedReader读取贴吧目录中的详细帖子地址
	 * 
	 * @param file
	 * @return
	 * @throws IOException
	 */
	public static String[] readFileLine(String file) throws IOException {
		BufferedReader br = null;
		String[] sb = new String[50];
		int i = 0;
		try {
			br = new BufferedReader(new FileReader(file));
			String line = null;

			while ((line = br.readLine()) != null) {
				if (line.contains("<a href=\"/p/")) {
					line = line.trim();
					line = "http://tieba.baidu.com" + line.substring(9, 22);
					sb[i] = line;
					i++;
				}
			}
		} catch (Exception ex) {
			System.out.println("Error occurs during reading " + file);
		} finally {
			if (br != null)
				br.close();
		}
		return sb;
	}

	/**
	 * 使用BufferedReader读取帖子详细内容中的图片链接地址
	 * 
	 * @param file
	 * @return
	 * @throws IOException
	 */
	public static String[] readDetailsFileLine(String file) throws IOException {
		BufferedReader br = null;
		String[] sb = new String[200];
		int i = 0;
		try {
			br = new BufferedReader(new FileReader(file));
			String line = null;

			while ((line = br.readLine()) != null) {

				if (line.contains("<li class=\"l_reply_num\" style=\"margin-left:2px; margin-right:10px\" >共有<span class=\"red\">")) {
					line = line
							.replace(
									"<li class=\"l_reply_num\" style=\"margin-left:2px; margin-right:10px\" >共有<span class=\"red\">",
									"");
					line = line.replace("</span>页</li>", "");
					line = line.trim();
					try {
						PageNumber = Integer.parseInt(line);
					} catch (Exception e) {
						PageNumber = 1;
					}
				}
				if (line.contains("http://imgsrc.baidu.com/forum")) {
					line = line.trim();
					String[] ss = line.split(" ");
					for (String s : ss) {

						if (s.contains("http://imgsrc.baidu.com/forum")) {
							String[] sss = s.split("\"");
							for (String s0 : sss) {
								if (s0.contains("http://imgsrc.baidu.com/forum")) {
									sb[i] = s0;
									i++;
								}
							}
						}
					}

				}
			}
		} catch (Exception ex) {
			System.out.println("有错误发生" + file);
		} finally {
			if (br != null)
				br.close();
		}
		return sb;
	}

	/**
	 * 提取URL中的文件名
	 * 
	 * @param url
	 * @return
	 */
	public static String getLastStringFromURL(String url) {
		try {
			String[] ss = url.split("/");
			int i = ss.length;
			ss[i - 1] = ss[i - 1].replace("?", "");
			ss[i - 1] = ss[i - 1].replace("v=tbs", "");
			return ss[i - 1];
		} catch (Exception e) {
			System.out.println(url);
		}
		return null;
	}

}
